{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0tT128fnePVL"
   },
   "source": [
    "掩码语言建模是用掩码标记对序列中的标记进行掩码，并提示模型用适当的标记填充该掩码的任务。这允许模型同时处理右上下文(掩码右侧的标记)和左上下文(掩码左侧的标记)。这样的训练为需要双向背景的下游任务(如SQuAD)奠定了坚实的基础。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 721
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12139,
     "status": "ok",
     "timestamp": 1586676491181,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "JCkraymzeaM3",
    "outputId": "1ab6323a-ceb6-44eb-9e4c-0e8bb5ee2f96"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/78/92cedda05552398352ed9784908b834ee32a0bd071a9b32de287327370b7/transformers-2.8.0-py3-none-any.whl (563kB)\n",
      "\r",
      "\u001b[K     |▋                               | 10kB 20.3MB/s eta 0:00:01\r",
      "\u001b[K     |█▏                              | 20kB 6.0MB/s eta 0:00:01\r",
      "\u001b[K     |█▊                              | 30kB 7.6MB/s eta 0:00:01\r",
      "\u001b[K     |██▎                             | 40kB 5.6MB/s eta 0:00:01\r",
      "\u001b[K     |███                             | 51kB 6.1MB/s eta 0:00:01\r",
      "\u001b[K     |███▌                            | 61kB 7.3MB/s eta 0:00:01\r",
      "\u001b[K     |████                            | 71kB 7.4MB/s eta 0:00:01\r",
      "\u001b[K     |████▋                           | 81kB 7.7MB/s eta 0:00:01\r",
      "\u001b[K     |█████▎                          | 92kB 8.6MB/s eta 0:00:01\r",
      "\u001b[K     |█████▉                          | 102kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████▍                         | 112kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████                         | 122kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████▋                        | 133kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████▏                       | 143kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████▊                       | 153kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▎                      | 163kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████▉                      | 174kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████▌                     | 184kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████                     | 194kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████▋                    | 204kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▏                   | 215kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████▉                   | 225kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████▍                  | 235kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████                  | 245kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████▌                 | 256kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▏                | 266kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▊                | 276kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▎               | 286kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████▉               | 296kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████▍              | 307kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████              | 317kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████▋             | 327kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▏            | 337kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████▊            | 348kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████▍           | 358kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████           | 368kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████▌          | 378kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████          | 389kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████▊         | 399kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▎        | 409kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▉        | 419kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████▍       | 430kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████       | 440kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████▋      | 450kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▏     | 460kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████▊     | 471kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████▎    | 481kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████    | 491kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████▌   | 501kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████   | 512kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |█████████████████████████████▋  | 522kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▎ | 532kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |██████████████████████████████▉ | 542kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▍| 552kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 563kB 8.5MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 573kB 8.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.2)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.38.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.12.38)\n",
      "Collecting tokenizers==0.5.2\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
      "\u001b[K     |████████████████████████████████| 3.7MB 32.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
      "Collecting sentencepiece\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
      "\u001b[K     |████████████████████████████████| 1.0MB 55.8MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
      "\u001b[K     |████████████████████████████████| 870kB 56.7MB/s \n",
      "\u001b[?25hRequirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.5)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
      "Requirement already satisfied: botocore<1.16.0,>=1.15.38 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.15.38)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.4.5.1)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.1)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->transformers) (0.15.2)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->transformers) (2.8.1)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=8024d170ab00e981972616660b8c1d521ca6229c86cb8f449fe9128fa71ac20d\n",
      "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
      "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.5.2 transformers-2.8.0\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12131,
     "status": "ok",
     "timestamp": 1586676491183,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "wuFU5Ua8eeol",
    "outputId": "6fc27531-b795-4647-ea57-0946200de36a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Tesla T4'"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 305,
     "referenced_widgets": [
      "3cb8eda247fe4ea3b8d059a264ca0082",
      "1a55bdffdb14424f8e2d2daec115267f",
      "d578d2397bae45f1b9e610122e47c564",
      "ac4c1ebc9e0e47d991282d32ae8dbbff",
      "030dd97d35044090a64d8a02adae94af",
      "55a5c6b2a7ac484f8d27b8c183b7e360",
      "bee49c74fe8741ff9c6d137e6bece381",
      "b79b07984a0f4127b05cb982cbef35fe",
      "41d91489776246ffa499a2d94d6c7f67",
      "b3c855209d544edb8f95e38a752824f7",
      "8b5008213ed54b5a95585f297941ef1b",
      "ce32527c747241e683c72f7b33621890",
      "b064591a7bd543feb86f2ed28428f243",
      "45fa93c46ece44ffbcdf6775c7eef3e5",
      "a723d0f1b9d44d86a5423e0156cc95b8",
      "df263f7a2a6e4a36aa2f9884e21b4220",
      "71bef6ef456a4a2a9c4a66d0cfea2afa",
      "2fbcbfbc475040a383a9056237db745f",
      "f4482e2719a14c0e93c06365e7aed3bd",
      "c471f4eca5984c7fa59175d460ff551d",
      "1cb14481aaa1469f80fdf3fef12c42dd",
      "b25eed703ba64904b130612c9c98eb09",
      "5b688e2bc52f409aaddfbedd683186ea",
      "ba6f95e8b55d4f3ba14c6fd8bf3baf3b",
      "e9e2bc4e49354f7b9dd1aedb16ef081f",
      "e0555818e8a5435f8fa80cfca25c23c2",
      "05fd9f3ca0104d9e896f8954040154fc",
      "567af497917045d4882d40c5e01f328b",
      "53e23311b686422281e0897ef82dac17",
      "5f52a2d80b7e42b4b355bfe34e1f74ad",
      "eeadab46bf6145358ac121d60d137ffd",
      "804f6922517249aa92e1ef0e91d8db41",
      "4dd9e05359844c9ca7cb2b2eef342af1",
      "6f32a19d22c84d94a23121c268407af0",
      "cb3dc96f62424a87bbc53e566df78d2f",
      "436051a0a8b84a9683d1c63fb97a855f",
      "800559fd6f054cff80ed33f09029523e",
      "28fca1fb25f04167b096c99aebf66bc3",
      "ae59dee3a4b447b58f551b370c29ad52",
      "649d7a6581144b9d8a1f27361bd14480"
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 33847,
     "status": "ok",
     "timestamp": 1586676512901,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "LbulATdieiVf",
    "outputId": "553ce770-ecca-4b67-ede8-bf731df229a1"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cb8eda247fe4ea3b8d059a264ca0082",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=523, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41d91489776246ffa499a2d94d6c7f67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=898823, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71bef6ef456a4a2a9c4a66d0cfea2afa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=456318, style=ProgressStyle(description_wid…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9e2bc4e49354f7b9dd1aedb16ef081f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=230, style=ProgressStyle(description_width=…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4dd9e05359844c9ca7cb2b2eef342af1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Downloading', max=331070498, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[{'sequence': '<s> HuggingFace is creating a tool that the community uses to solve NLP tasks.</s>', 'score': 0.1572076976299286, 'token': 3944}, {'sequence': '<s> HuggingFace is creating a framework that the community uses to solve NLP tasks.</s>', 'score': 0.11565146595239639, 'token': 7208}, {'sequence': '<s> HuggingFace is creating a library that the community uses to solve NLP tasks.</s>', 'score': 0.05949191749095917, 'token': 5560}, {'sequence': '<s> HuggingFace is creating a database that the community uses to solve NLP tasks.</s>', 'score': 0.04147905111312866, 'token': 8503}, {'sequence': '<s> HuggingFace is creating a prototype that the community uses to solve NLP tasks.</s>', 'score': 0.025827907025814056, 'token': 17715}]\n"
     ]
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "nlp = pipeline(\"fill-mask\")\n",
    "print(nlp(f\"HuggingFace is creating a {nlp.tokenizer.mask_token} that the community uses to solve NLP tasks.\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Yd2cQZtWSxIW"
   },
   "source": [
    "### 下面是一个使用模型和Tokenizer进行掩码语言建模的示例。该过程如下：\n",
    "\n",
    "– 从checkpoint名称实例化一个tokenizer和一个模型。该模型被识别为一个DistilBERT模型，并用存储在checkpoint中的权重加载它。\n",
    "\n",
    "– 定义一个带掩码标记的序列，不使用单词而是选择`tokenizer.mask_token`进行放置(进行掩码)。\n",
    "\n",
    "– 将该序列编码为id，并在该id列表中找到掩码标记的位置。\n",
    "\n",
    "– 在掩码标记的索引处检索预测：此张量与词汇表的大小相同，值是每个标记的分数。模型对他认为在这种情况下可能出现的标记会给出更高的分数。\n",
    "\n",
    "– 使用PyTorch `topk`方法检索前5个标记。\n",
    "\n",
    "– 用预测的标记替换掩码标记并打印结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4307,
     "status": "ok",
     "timestamp": 1586677320013,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "oxNb3mViTSAw",
    "outputId": "f45c8b2d-04e8-4f7b-8a98-13ed269d7d65"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[  101, 12120,  2050,  8683,  1181,  3584,  1132,  2964,  1190,  1103,\n",
      "          3584,  1152, 27180,   119,  7993,  1172,  1939,  1104,  1103,  1415,\n",
      "          3827,  1156,  1494,   103,  1412,  6302,  2555, 10988,   119,   102]])\n",
      "103\n",
      "(tensor([0]), tensor([23]))\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelWithLMHead, AutoTokenizer\n",
    "import torch\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-cased\")\n",
    "model = AutoModelWithLMHead.from_pretrained(\"distilbert-base-cased\")\n",
    "\n",
    "sequence = f\"Distilled models are smaller than the models they mimic. Using them instead of the large versions would help {tokenizer.mask_token} our carbon footprint.\"\n",
    "\n",
    "input = tokenizer.encode(sequence, return_tensors=\"pt\")\n",
    "mask_token_index = torch.where(input == tokenizer.mask_token_id)[1]#返回了103的位置\n",
    "\n",
    "print(input)\n",
    "print(tokenizer.mask_token_id)\n",
    "print(torch.where(input == tokenizer.mask_token_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 179
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1257,
     "status": "ok",
     "timestamp": 1586677441312,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "_otgMRbgT-Td",
    "outputId": "9e15760f-6adc-4cbc-a70d-f0a25dd0c339"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[ -6.6732,  -6.6450,  -6.7923,  ...,  -5.5930,  -5.2783,  -5.6559],\n",
      "         [ -6.3221,  -5.6379,  -5.8990,  ...,  -4.6864,  -4.1499,  -5.3507],\n",
      "         [ -5.9863,  -6.0991,  -5.8089,  ...,  -5.2297,  -4.3015,  -6.5971],\n",
      "         ...,\n",
      "         [ -7.8892,  -7.6718,  -7.6357,  ...,  -6.9083,  -5.5853,  -6.2459],\n",
      "         [-14.7710, -14.2714, -14.1642,  ..., -11.4770, -12.1692, -13.1041],\n",
      "         [-14.3694, -13.9838, -13.6330,  ..., -11.2066, -11.6753, -12.7083]]],\n",
      "       grad_fn=<AddBackward0>)\n",
      "tensor([23])\n"
     ]
    }
   ],
   "source": [
    "token_logits = model(input)[0]\n",
    "mask_token_logits = token_logits[0, mask_token_index, :]\n",
    "\n",
    "print(token_logits)\n",
    "print(mask_token_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "r-kNkxCjT-jQ"
   },
   "outputs": [],
   "source": [
    "top_5_tokens = torch.topk(mask_token_logits, 5, dim=1).indices[0].tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 750,
     "status": "ok",
     "timestamp": 1586677506836,
     "user": {
      "displayName": "Xue Jiang",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjfXAo9TmWFYS-GhDq8HOVk3N1FnJiyBPH8h9M=s64",
      "userId": "12544639825510830792"
     },
     "user_tz": -480
    },
    "id": "o-n4qnPhUCtH",
    "outputId": "f727473d-5968-4022-a553-9d83e2569a92"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4851\n",
      "Distilled models are smaller than the models they mimic. Using them instead of the large versions would help reduce our carbon footprint.\n",
      "2773\n",
      "Distilled models are smaller than the models they mimic. Using them instead of the large versions would help increase our carbon footprint.\n",
      "9711\n",
      "Distilled models are smaller than the models they mimic. Using them instead of the large versions would help decrease our carbon footprint.\n",
      "18134\n",
      "Distilled models are smaller than the models they mimic. Using them instead of the large versions would help offset our carbon footprint.\n",
      "4607\n",
      "Distilled models are smaller than the models they mimic. Using them instead of the large versions would help improve our carbon footprint.\n"
     ]
    }
   ],
   "source": [
    "for token in top_5_tokens:\n",
    "  print(token)\n",
    "  print(sequence.replace(tokenizer.mask_token, tokenizer.decode([token])))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyOQ+B8rvEvWRVjxwHU8WN+j",
   "collapsed_sections": [],
   "name": "掩码语言建模.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "030dd97d35044090a64d8a02adae94af": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "05fd9f3ca0104d9e896f8954040154fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5f52a2d80b7e42b4b355bfe34e1f74ad",
      "max": 230,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_53e23311b686422281e0897ef82dac17",
      "value": 230
     }
    },
    "1a55bdffdb14424f8e2d2daec115267f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1cb14481aaa1469f80fdf3fef12c42dd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "28fca1fb25f04167b096c99aebf66bc3": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2fbcbfbc475040a383a9056237db745f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3cb8eda247fe4ea3b8d059a264ca0082": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_d578d2397bae45f1b9e610122e47c564",
       "IPY_MODEL_ac4c1ebc9e0e47d991282d32ae8dbbff"
      ],
      "layout": "IPY_MODEL_1a55bdffdb14424f8e2d2daec115267f"
     }
    },
    "41d91489776246ffa499a2d94d6c7f67": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8b5008213ed54b5a95585f297941ef1b",
       "IPY_MODEL_ce32527c747241e683c72f7b33621890"
      ],
      "layout": "IPY_MODEL_b3c855209d544edb8f95e38a752824f7"
     }
    },
    "436051a0a8b84a9683d1c63fb97a855f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_649d7a6581144b9d8a1f27361bd14480",
      "placeholder": "​",
      "style": "IPY_MODEL_ae59dee3a4b447b58f551b370c29ad52",
      "value": " 331M/331M [00:14&lt;00:00, 23.5MB/s]"
     }
    },
    "45fa93c46ece44ffbcdf6775c7eef3e5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4dd9e05359844c9ca7cb2b2eef342af1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cb3dc96f62424a87bbc53e566df78d2f",
       "IPY_MODEL_436051a0a8b84a9683d1c63fb97a855f"
      ],
      "layout": "IPY_MODEL_6f32a19d22c84d94a23121c268407af0"
     }
    },
    "53e23311b686422281e0897ef82dac17": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "55a5c6b2a7ac484f8d27b8c183b7e360": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "567af497917045d4882d40c5e01f328b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_804f6922517249aa92e1ef0e91d8db41",
      "placeholder": "​",
      "style": "IPY_MODEL_eeadab46bf6145358ac121d60d137ffd",
      "value": " 230/230 [00:01&lt;00:00, 195B/s]"
     }
    },
    "5b688e2bc52f409aaddfbedd683186ea": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5f52a2d80b7e42b4b355bfe34e1f74ad": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "649d7a6581144b9d8a1f27361bd14480": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6f32a19d22c84d94a23121c268407af0": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "71bef6ef456a4a2a9c4a66d0cfea2afa": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f4482e2719a14c0e93c06365e7aed3bd",
       "IPY_MODEL_c471f4eca5984c7fa59175d460ff551d"
      ],
      "layout": "IPY_MODEL_2fbcbfbc475040a383a9056237db745f"
     }
    },
    "800559fd6f054cff80ed33f09029523e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "804f6922517249aa92e1ef0e91d8db41": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8b5008213ed54b5a95585f297941ef1b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_45fa93c46ece44ffbcdf6775c7eef3e5",
      "max": 898823,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b064591a7bd543feb86f2ed28428f243",
      "value": 898823
     }
    },
    "a723d0f1b9d44d86a5423e0156cc95b8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ac4c1ebc9e0e47d991282d32ae8dbbff": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b79b07984a0f4127b05cb982cbef35fe",
      "placeholder": "​",
      "style": "IPY_MODEL_bee49c74fe8741ff9c6d137e6bece381",
      "value": " 523/523 [00:02&lt;00:00, 206B/s]"
     }
    },
    "ae59dee3a4b447b58f551b370c29ad52": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b064591a7bd543feb86f2ed28428f243": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b25eed703ba64904b130612c9c98eb09": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b3c855209d544edb8f95e38a752824f7": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b79b07984a0f4127b05cb982cbef35fe": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ba6f95e8b55d4f3ba14c6fd8bf3baf3b": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bee49c74fe8741ff9c6d137e6bece381": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c471f4eca5984c7fa59175d460ff551d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ba6f95e8b55d4f3ba14c6fd8bf3baf3b",
      "placeholder": "​",
      "style": "IPY_MODEL_5b688e2bc52f409aaddfbedd683186ea",
      "value": " 456k/456k [00:00&lt;00:00, 1.10MB/s]"
     }
    },
    "cb3dc96f62424a87bbc53e566df78d2f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_28fca1fb25f04167b096c99aebf66bc3",
      "max": 331070498,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_800559fd6f054cff80ed33f09029523e",
      "value": 331070498
     }
    },
    "ce32527c747241e683c72f7b33621890": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_df263f7a2a6e4a36aa2f9884e21b4220",
      "placeholder": "​",
      "style": "IPY_MODEL_a723d0f1b9d44d86a5423e0156cc95b8",
      "value": " 899k/899k [00:01&lt;00:00, 680kB/s]"
     }
    },
    "d578d2397bae45f1b9e610122e47c564": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_55a5c6b2a7ac484f8d27b8c183b7e360",
      "max": 523,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_030dd97d35044090a64d8a02adae94af",
      "value": 523
     }
    },
    "df263f7a2a6e4a36aa2f9884e21b4220": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e0555818e8a5435f8fa80cfca25c23c2": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e9e2bc4e49354f7b9dd1aedb16ef081f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_05fd9f3ca0104d9e896f8954040154fc",
       "IPY_MODEL_567af497917045d4882d40c5e01f328b"
      ],
      "layout": "IPY_MODEL_e0555818e8a5435f8fa80cfca25c23c2"
     }
    },
    "eeadab46bf6145358ac121d60d137ffd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f4482e2719a14c0e93c06365e7aed3bd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b25eed703ba64904b130612c9c98eb09",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_1cb14481aaa1469f80fdf3fef12c42dd",
      "value": 456318
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
